import json
import datasets
from datasets import load_dataset

dataset = load_dataset("logistic_data", split="train",trust_remote_code=True)
print(dataset[0:40])
dataset = load_dataset("dataset_handler.py", split="train",trust_remote_code=True)
print(dataset[0:40])
# sdataset = dataset.train_test_split(test_size=0.1)
# train_dataset = sdataset["train"]
# val_dataset = sdataset["test"]
#
# # 将数据集以 JSON Lines 格式保存到文件，保持中文字符原样
# with open('train_dataset.json', 'w', encoding='utf-8') as f:
#     for example in train_dataset:
#         json_str = json.dumps(example, ensure_ascii=False)  # 注意这里添加了 ensure_ascii=False
#         f.write(json_str + '\n')
#
# with open('val_dataset.json', 'w', encoding='utf-8') as f:
#     for example in val_dataset:
#         json_str = json.dumps(example, ensure_ascii=False)  # 注意这里添加了 ensure_ascii=False
#         f.write(json_str + '\n')
#
# print(len(train_dataset))
# print(len(val_dataset))

# train_dataset.to_json("train_dataset.json",mode="w")
# val_dataset.to_json("val_dataset.json",mode="w")